import qp
from yos.boss import ysearch
from yos.yql import db
import nltk

QTYPE_POINT=100
KEYWORD_COUNT_POINT=8
KEYWORD_ORDER_POING=5
KEYWORD_VARIATION_POINT=2
MAX_WORD_IN_SENT=1000

def search_web(orig_query,ques_point):
	
	count_of_result=30
	entity_choice_list=[]
	
	data=ysearch.search(orig_query,count=count_of_result,more={'abstract':'long'})
	table=db.create(data=data)
	
	keyword_list=get_keywords(orig_query)
	print "keywords=",keyword_list
	count=0
	
	for r in table.rows:
		title=nltk.clean_html(r['title'])
		clickurl=nltk.clean_html(r['clickurl'])
		abstract=nltk.clean_html(r['abstract'])
		#call process_result -> it will store results according to point in entity_choice_list
		process_result(entity_choice_list,keyword_list,ques_point,title,clickurl,abstract)

	#now return the list	
	return entity_choice_list


#need to modify in LARGE!!!!!!!!!!!!!!!!!!!!	
def process_result(entity_choice_list,keyword_list,ques_point,title,clickurl,abstract):
	""" write documentation """
	abs_sents=nltk.sent_tokenize(abstract)
	for sent in abs_sents:
		sent_toks=nltk.word_tokenize(sent)
		sent_tags=nltk.pos_tag(sent_toks)
		sent_ne=nltk.ne_chunk(sent_tags)
		sent_subtrees=sent_ne.subtrees()
		
		#now check if there are any subtrees, except the first one
		#and see it it has the ANSWER_TYPE
		check=0
		word=[]
		for i in sent_subtrees:
			if check==0:
				check=1
			else:
				if ques_point==i.node:
					word=get_word_as_list(i.leaves())
					#now add it to entity_choice_list, along with score
					
					#check if the word is already present in keyword_list
					if compare_word_list(word,keyword_list)==0:
						#now see the no of keywords
						k_point=get_score(sent_toks,keyword_list)
					
						#now check if duplicate entry added, or not
						index_position=entity_already_exist(entity_choice_list,word)
						if index_position== -1:
							entity_choice_list.append([word,k_point,1])
						else:
							add_dup_entity_choice_list_entry(entity_choice_list,word,k_point,index_position)
	
	
	
	return
	
def get_score(sent_toks, keyword_list):
	""" returns a score based on:
	1.	no of keywords
	2.	order of keywords
	3. 	sparseness of keywords
	"""
	k_point=0
	
	#the var 'keywords_present' is of the form [[key1,loc],...]
	keywords_present=get_keywords_present(sent_toks, keyword_list)
	no_of_keywords_present=len(keywords_present)
	no_of_keywords=len(keyword_list)
	
	#first point calculation
	k_point+=no_of_keywords_present*KEYWORD_COUNT_POINT
	
	#second point calculation
	order_list=[]
	for k in keywords_present:
		order_list.append(k[1])
	
	order_count= get_max_no_keys_in_order(order_list)
	order_weight=(float(order_count)/no_of_keywords/4)
	p_to_increase=k_point*order_weight
	k_point+=round(p_to_increase)
	
	#third point calculation
	
	
	return k_point
	
	
def get_keywords_present(sent_toks,keyword_list):
	"""returns a dictionary of the form [[key1,loc],...]
	"""
	keywords_present=[]
	for i in keyword_list:
		count=0
		for j in sent_toks:
			if i.lower() == j.lower() :
				keywords_present.append([i,count])
				break
			count+=1
	
	return keywords_present
	
	
	
def get_max_no_keys_in_order(order_list):
	"""Returns no of keys in order as given in the question
	"""
	max_list=[]
	for i in order_list:
		tmp_list=[i]
		for j in order_list[order_list.index(i)+1:]:
			if j > tmp_list[-1]:
				tmp_list.append(j)
		if len(tmp_list)> len(max_list):
			max_list=tmp_list
	
	return len(max_list)


	
def entity_already_exist(entity_choice_list,word):
	"""checks if 'word' or its variants already exist in the 'entity_choice_list' """
	count=0
	for w in entity_choice_list:
		actual_word=w[0]
		#now try to match any similiarities between word and actual_word
		if compare_word_list(actual_word,word)==1:
			return count
		else:
			count+=1
			
	return -1


def compare_word_list(actual_word,word):
	"""tries to find any matches between two lists, and returns 1 if matches exist, else 0"""
	flag=0
	for w in word:
		for i in actual_word:
			if w.lower() == i.lower() :
				flag=1
				break
	
	return flag
	
	
def add_dup_entity_choice_list_entry(entity_choice_list,word,k_point,index_position):
	""" modify the entity_choice_list by adding the new point gained"""
	dup_entry=entity_choice_list[index_position]
	existing_point=dup_entry[1]
	existing_word=dup_entry[0]
	existing_count=dup_entry[2]
	
	if len(existing_word)< len(word) :
		new_entry=[word,existing_point+k_point,existing_count+1]
	else:
		new_entry=[existing_word,existing_point+k_point,existing_count+1]
	
	#now change the entry
	entity_choice_list[index_position]=new_entry
	
	
	
#to be modified to return proper keywords	
def get_keywords(orig_query):
	keyword_tag_list=["CD","JJ","JJR","JJS","LS","NN","NNP","NNPS","NNS"]
	(new_query,modify_list)=qp.modify_query(orig_query)
	toks=nltk.word_tokenize(new_query)
	tags=nltk.pos_tag(toks)
	keyword_list=[]
	for i in tags:
		if i[1] in keyword_tag_list:
			if modify_list.has_key(i[0]):
				keyword_list.append(modify_list[i[0]])
			else:
				keyword_list.append(i[0])
	
	return keyword_list

	
def get_word_as_list(alist):
	wd=[]
	for i in alist:
		wd.append(i[0])
	return wd

	
